# -*- coding:utf-8 -*-
'''
Created on Mar 15, 2015

@author: eyaomai
'''
import sys
sys.path.append(sys.argv[0][:sys.argv[0].rfind('com/stocklens')])

from com.stocklens.stock.common.utils import Logging,  PIDUtils 
from com.stocklens.stock.data.crawlercommon import CrawlerConstants
from com.stocklens.stock.data.mas.mascommon import MASConstant, MASCrawlerManager,MASCrawlerBase
from bs4 import BeautifulSoup
import traceback

class MOFCrawlerManager(MASCrawlerManager):
    LOGGER_NAME_CRAWL_MOF = 'mof'
    MINIMAL_THREAD_NUM = 9

    def __init__(self, json_config_file):
        '''
        Constructor
        '''
        super(MOFCrawlerManager, self).__init__(json_config_file, MOFCrawlerManager.MINIMAL_THREAD_NUM)
        self._masTitle = '科技部'
        self.logger = Logging.getLogger(MOFCrawlerManager.LOGGER_NAME_CRAWL_MOF)

    def _initTask(self):
        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/c/',
                   MASConstant.PARA_PREFIX:'部令公告'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/fwzl/',
                   MASConstant.PARA_PREFIX:'综合政策'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/d/',
                   MASConstant.PARA_PREFIX:'国内贸易管理'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/e/',
                   MASConstant.PARA_PREFIX:'对外贸易管理'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/f/',
                   MASConstant.PARA_PREFIX:'外国投资管理'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/bf/',
                   MASConstant.PARA_PREFIX:'国外经济合作'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/xxfb/',
                   MASConstant.PARA_PREFIX:'服务贸易'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/b/g/',
                   MASConstant.PARA_PREFIX:'对外经贸文告'}
        self._taskList.append(request)        

        request = {CrawlerConstants.PARA_CLASS:ArticleCrawler,
                   MASConstant.PARA_URL:'http://www.mofcom.gov.cn/article/ae/ai/',
                   MASConstant.PARA_PREFIX:'日常新闻'}
        self._taskList.append(request)        

class ArticleCrawler(MASCrawlerBase):
    MAX_ITEMS = 40
    def __init__(self, controller, dbProxy, request):
        super(ArticleCrawler, self).__init__(controller, 
                                                   dbProxy, 
                                                   request, 
                                                   ArticleCrawler.MAX_ITEMS, 
                                                   MOFCrawlerManager.LOGGER_NAME_CRAWL_MOF)

    def _crawl(self):
        super(ArticleCrawler, self)._crawl()
        try:
            content = self._fetchContent(self._url, None, False)
            if content is None:
                return
            soup = BeautifulSoup(content)
            divs = soup.findAll('div',{'class':'listBox borB'})
            divLen = len(divs)
            if divLen==0:
                self.logger.warn('div not found. Format may have change or anti-crawled')
                return
            div = divs[0]
            ahrefs = div.findAll('a')
            for ahref in ahrefs:
                title = ahref['title']
                if ahref['href'].startswith('/'):
                    link = self._url+ahref['href'][1:]
                else:
                    link = self._url+ahref['href']
                releaseDate = ahref.fetchNextSiblings()[0].text
                self._handleItem(releaseDate, title, link)
        except Exception:
            traceInfo = traceback.format_exc()
            self.logger.warn('Fail to crawl:%s', traceInfo)    
    

if __name__ == '__main__':
    if PIDUtils.isPidFileExist('mof'):
        print 'Previous MOFCom Crawler process is on-going, please stop it firstly'
        sys.exit(1)
    import os
    pid = os.getpid()
    PIDUtils.writePid('mof', pid)
    Logging.initLogger('conf/crawler/crawler.logging.cfg')
    mof = MOFCrawlerManager('conf/crawler/mof.cfg')
    mof.start()
    pidutils = PIDUtils('mof', mof.shutDown, 5, mof.logger)
    pidutils.start()
    sys.exit(0)
        